{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- PE(pos,2_i) = sin(pos/10000^2i/d_model)\n",
    "- PE(pos,2_i+1) = cos(pos/10000^2i/d_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.73912054904549596"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sin(1/10000**(1/dim))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positional_encoding1(sentence_length, dim):\n",
    "    encoded_vec = []\n",
    "    for pos in range(sentence_length):\n",
    "        for i in range(dim):\n",
    "            vec = pos/10000**(2*i/dim)\n",
    "            if \n",
    "            \n",
    "            if i % 2 == 0:\n",
    "                vec = np.sin(vec)\n",
    "            else:\n",
    "                vec = np.cos(vec)\n",
    "            encoded_vec.append(vec)\n",
    "    return np.array(encoded_vec).reshape([sentence_length, dim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 20)\n",
      "[[  0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00]\n",
      " [  8.41470985e-01   9.21796446e-01   1.57826640e-01   9.98010124e-01\n",
      "    2.51162229e-02   9.99950000e-01   3.98106119e-03   9.99998744e-01\n",
      "    6.30957303e-04   9.99999968e-01   9.99999998e-05   9.99999999e-01\n",
      "    1.58489319e-05   1.00000000e+00   2.51188643e-06   1.00000000e+00\n",
      "    3.98107171e-07   1.00000000e+00   6.30957344e-08   1.00000000e+00]\n",
      " [  9.09297427e-01   6.99417376e-01   3.11697146e-01   9.92048417e-01\n",
      "    5.02165994e-02   9.99800007e-01   7.96205928e-03   9.99994976e-01\n",
      "    1.26191435e-03   9.99999874e-01   1.99999999e-04   9.99999997e-01\n",
      "    3.16978638e-05   1.00000000e+00   5.02377286e-06   1.00000000e+00\n",
      "    7.96214341e-07   1.00000000e+00   1.26191469e-07   1.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "pe = positional_encoding1(10, 20)\n",
    "\n",
    "print(pe.shape)\n",
    "print(pe[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positional_encoding2(sentence_length, dim):\n",
    "    encoded_vec = np.array([pos/10000**(2*i/dim) for pos in range(sentence_length) for i in range(dim)])\n",
    "    \n",
    "    encoded_vec[::2] = np.sin(encoded_vec[::2])\n",
    "    encoded_vec[1::2] = np.cos(encoded_vec[1::2])\n",
    "    \n",
    "    return np.array(encoded_vec).reshape([sentence_length, dim])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "(10, 20)\n",
      "[[  0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00\n",
      "    0.00000000e+00   1.00000000e+00   0.00000000e+00   1.00000000e+00]\n",
      " [  8.41470985e-01   9.21796446e-01   1.57826640e-01   9.98010124e-01\n",
      "    2.51162229e-02   9.99950000e-01   3.98106119e-03   9.99998744e-01\n",
      "    6.30957303e-04   9.99999968e-01   9.99999998e-05   9.99999999e-01\n",
      "    1.58489319e-05   1.00000000e+00   2.51188643e-06   1.00000000e+00\n",
      "    3.98107171e-07   1.00000000e+00   6.30957344e-08   1.00000000e+00]\n",
      " [  9.09297427e-01   6.99417376e-01   3.11697146e-01   9.92048417e-01\n",
      "    5.02165994e-02   9.99800007e-01   7.96205928e-03   9.99994976e-01\n",
      "    1.26191435e-03   9.99999874e-01   1.99999999e-04   9.99999997e-01\n",
      "    3.16978638e-05   1.00000000e+00   5.02377286e-06   1.00000000e+00\n",
      "    7.96214341e-07   1.00000000e+00   1.26191469e-07   1.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "pe = positional_encoding2(10, 20)\n",
    "\n",
    "print(pe.shape)\n",
    "print(pe[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          1.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          1.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          0.00000000e+00,   1.00000000e+00,   0.00000000e+00,\n",
       "          1.00000000e+00,   0.00000000e+00,   1.00000000e+00,\n",
       "          0.00000000e+00,   1.00000000e+00],\n",
       "       [  8.41470985e-01,   9.21796446e-01,   1.57826640e-01,\n",
       "          9.98010124e-01,   2.51162229e-02,   9.99950000e-01,\n",
       "          3.98106119e-03,   9.99998744e-01,   6.30957303e-04,\n",
       "          9.99999968e-01,   9.99999998e-05,   9.99999999e-01,\n",
       "          1.58489319e-05,   1.00000000e+00,   2.51188643e-06,\n",
       "          1.00000000e+00,   3.98107171e-07,   1.00000000e+00,\n",
       "          6.30957344e-08,   1.00000000e+00],\n",
       "       [  9.09297427e-01,   6.99417376e-01,   3.11697146e-01,\n",
       "          9.92048417e-01,   5.02165994e-02,   9.99800007e-01,\n",
       "          7.96205928e-03,   9.99994976e-01,   1.26191435e-03,\n",
       "          9.99999874e-01,   1.99999999e-04,   9.99999997e-01,\n",
       "          3.16978638e-05,   1.00000000e+00,   5.02377286e-06,\n",
       "          1.00000000e+00,   7.96214341e-07,   1.00000000e+00,\n",
       "          1.26191469e-07,   1.00000000e+00]])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_length = 10\n",
    "dim = 20\n",
    "\n",
    "encoded_vec = np.array([pos/10000**(2*i/dim) for pos in range(sentence_length) for i in range(dim)])\n",
    "\n",
    "encoded_vec[::2] = np.sin(encoded_vec[::2])\n",
    "encoded_vec[1::2] = np.cos(encoded_vec[1::2])\n",
    "    \n",
    "encoded_vec.reshape([sentence_length, dim])[:3]\n",
    "# using convert to tensor"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6 (NLP)",
   "language": "python",
   "name": "nlp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
